# -*- coding: UTF-8 -*-
"""
Created on 2018年5月5日
@author: Leo
"""

# 系统库
import hashlib
from datetime import datetime
from collections import OrderedDict

# 第三方库
import scrapy


class OpenChinaSpider(scrapy.Spider):
    name = "OpenChina"

    def __init__(self, **kwargs):
        # 字段
        self.search_text = kwargs['text']
        self.page = kwargs['total_page']

        # 启动页面
        self.start_urls = []
        for page in range(1, int(self.page) + 1):
            u = "https://www.oschina.net/search?scope=blog&q={}&fromerr=gXlCN1f6&p={}".format(self.search_text, page)
            self.start_urls.append(u)

        # 数据有序字典
        self._data_od = OrderedDict()

        super().__init__(**kwargs)

    # 请求层
    def start_requests(self):
        for req_url in self.start_urls:
            yield scrapy.Request(url=req_url,
                                 method="GET",
                                 callback=self.parse)

    def parse(self, response):
        li_s = response.xpath('//ul[@id="results"]/li')

        for li in li_s:
            self._data_od = OrderedDict()
            title_tag = li.xpath('string(h3/span[@class="icon"])').extract_first()
            if title_tag != "原":
                continue
            else:
                self._data_od['title'] = li.xpath('string(h3/a)').extract_first()
                self._data_od['href'] = li.xpath('string(p[@class="url"])').extract_first()
                info = li.xpath('string(p[@class="date"])').extract_first().replace("\n", "").split("\t")[2].split(" ")
                self._data_od['createDate'] = datetime.strptime(info[0], "%Y-%m-%d")
                self._data_od['authorId'] = info[-1].replace("@", "")
                self._data_od['content'] = ''
                self._data_od['tag'] = self.search_text
                self._data_od['visitorsIds'] = []
                self._data_od['visitorsNum'] = 0
                """
                 类型（atype）共有三种分别是：
                 0：学习路线推荐文章
                 1：知识点推荐文章
                 2：知识点必读文章（教师发布的）
                """
                self._data_od['atype'] = 1
                self._data_od['knowledgePointId'] = ""
                self._data_od['knowledgePointName'] = ""
                self._data_od['sectionName'] = ""
                self._data_od['courseName'] = ""
                self._data_od['origin'] = "开源中国"
                try:
                    self._data_od['titleMD5'] = \
                        hashlib.md5(self._data_od['title'].encode(encoding='gb2312')).hexdigest()
                except UnicodeEncodeError:
                    self._data_od['titleMD5'] = \
                        hashlib.md5(self._data_od['title'].encode(encoding='utf-8')).hexdigest()
                self._data_od['contentMD5'] = ""
                self._data_od['recommendedId'] = ""
                self._data_od['remark'] = ""
                # print(dict(self._data_od))
                yield self._data_od
        # print("############################################################################################")
